from ma.commons.core.newjobaction import NewJobAction
from ma.commons.core.newmaptaskaction import NewMapTaskAction
from ma.commons.core.newreducetaskaction import NewReduceTaskAction
from ma.commons.core.constants import * 

import logging
import logging.config
import ma.const
import ma.log
from ma.fs.dfs.dfsflags import *
from .taskscheduler import TaskScheduler
from ma.commons.core.completejobsaction import CompleteJobsAction
from ma.fs.dfs._hdfs import HDFS
import time
import sys
import os
import os.path

    
class HDFSInteracter(object):
    """this class handles a  NodeTracker interaction with the HDFS
    """
    
    def __init__(self, nodetracker, forced_job_id):
        
        """the ctor
        """
        self.a = 1
        
        #initializing the logger
        self.__log = ma.log.get_logger('ma.mrimprov')
        
        # this reference to the HDFS is only for interacting with the DFS flags
        host = ma.const.XmlData.get_str_data(ma.const.xml_hdfs_host)
        port = ma.const.XmlData.get_int_data(ma.const.xml_hdfs_port)
        self.hdfs  = HDFS(host, port)
        
        # when using this
        #self.hdfs = nodetracker.hdfs
        
        self.dfsflags = DfsFlags(self.hdfs)
        
        if forced_job_id != None:
            self.__log.warning('Forcing job id: %d', forced_job_id)
            self.dfsflags.force_job_id(forced_job_id)
            
        #this is the task scheduler that suggests what job to be picked up next
        self.task_scheduler = TaskScheduler(self.dfsflags.get_active_job_list(), nodetracker)
        
        # nodetracker reference
        self.__nt = nodetracker
    
    
    def updateJobsListAtNT(self, actions):
        
        #list of active jobs got from the HDFS
        list_of_jobs = self.dfsflags.get_active_job_list()
        
        self.__log.debug('Updating jobs list: %s - TODO: Get actual list from HDFS', str(list_of_jobs))
        
        #passing the active jobs list to update the scheduler's own job
        #repository; returns list of done or killed jobs
        inactive_jobs = self.task_scheduler.refreshJobsInfo(list_of_jobs)
       
        index = 0
        list = []
        #this loop will add CompleteJobAction objects to the actions list to
        #be sent to the NT for housekeeping
        while index < len(inactive_jobs):
            list.append(inactive_jobs[index])
            done_job = CompleteJobsAction(list)
            actions.append(done_job)
            index += 1
            
        
        
    def heartbeatFromNT(self, taskinfo_list, tasks_to_run):
        
        """ This function will act as the master and interact with the HDFS on behalf of the 
        NT
        """
        
        self.__log.info("<3 <3 <3 HEARTBEAT from NT called")
        #list of actions to be returned
        actions = []
        
        #updating the job list from the HDFS
        self.updateJobsListAtNT(actions)
        
        dict_of_complete_maps_by_job_id = {}
        dict_of_complete_reduces_by_job_id = {}
               
        #placing complete and failed tasks info from the NT onto the hDFS
        for i in taskinfo_list:
            if i.type == COMPLETEDTASK_INFO_TYPE:
                
                #mark task as complete on the HDFS
                if i.map_or_reduce == MAP:
                    
                    if i.job_id not in dict_of_complete_maps_by_job_id:
                        dict_of_complete_maps_by_job_id[i.job_id] = []
                    dict_of_complete_maps_by_job_id[i.job_id].append((i.task_id,i.struct_id))
                    
                    self.__log.debug("map task %d with job_id %d reported by NT as complete", i.task_id, i.job_id)
                else:
                    #self.dfsflags.mark_reduces_complete(i.job_id, i.task_id)
                    #list of tuples (reduce_id, struct_id, level)
                    
                    if i.job_id not in dict_of_complete_reduces_by_job_id:
                        dict_of_complete_reduces_by_job_id[i.job_id] = []
                    dict_of_complete_reduces_by_job_id[i.job_id].append((i.task_id,i.struct_id))
                    
                    self.__log.debug("reduce task %d with job_id %d reported by NT as complete", i.task_id, i.job_id)
                
            elif i.type == FAILEDTASK_INFO_TYPE:
                
                # TODO: need to write HDFS code for failing a task
                if i.map_or_reduce == MAP:
                    self.__log.error("map task %d with job_id %d reported by NT as failed : TODO Code failed task on HDFS", i.task_id, i.job_id)
                else:
                    self.__log.error("reduce task %d with job_id %d reported by NT as failed : TODO Code failed task on HDFS", i.task_id, i.job_id)
                    
                """
                
                #mark task as failed on the HDFS
                if i.map_or_reduce == MAP:
                    self.dfsflags.mark_map_failed(i.job_id, i.task_id)
                    self.__log.debug("A map task with job_id %d " %job_id)
                    self.__log.debug("and task_id %d reported as failed" %i.task_id)
                else:
                    self.dfsflags.mark_reduce_failed(i.job_id, i.task_id)
                    self.__log.debug("A reduce task with job_id %d " %job_id)
                    self.__log.debug("and task_id %d reported as failed" %i.task_id)
                """
        for key in dict_of_complete_maps_by_job_id:
            ret = self.dfsflags.mark_maps_complete(dict_of_complete_maps_by_job_id[key],key)
            while ret == None:
                time.sleep(0.1)
                ret = self.dfsflags.mark_maps_complete(dict_of_complete_maps_by_job_id[key],key)
            if ret == True:
                self.task_scheduler.markMapsComplete(key) 
             
        for key in dict_of_complete_reduces_by_job_id:
            ret = self.dfsflags.mark_reduces_complete(dict_of_complete_reduces_by_job_id[key],key)
            while ret == None:
                time.sleep(0.1)
                ret = self.dfsflags.mark_reduces_complete(dict_of_complete_reduces_by_job_id[key],key)
        
        #assigning tasks to the NT according to the number of fresh tasks it can run
        self.assignNewTasksToNT(tasks_to_run, actions)
        
        return actions    
        
    
    def finalOuputCopyToDfs(self, job_id, reduce_phase_completed):
        """This function copies the final output to the DFS
        """
        self.__log.info("--> Attempting to copy Reduce outputs to DFS: %s", str(reduce_phase_completed))
        output_dest_dir = ma.const.JobsXmlData.get_filepath_str_data(ma.const.xml_local_output_temp_dir, job_id)
        dfs_output_dir = ma.const.JobsXmlData.get_dfs_filepath_str_data(ma.const.xml_dfs_path_job_output, job_id)

        for struct_id in reduce_phase_completed:
            # get the reduce id
            reduce_id = reduce_phase_completed[struct_id]
            # output filename
            filename = ma.const.JobsXmlData.get_str_data(ma.const.xml_reduce_output_filename, job_id, reduce_id)
            # form the fileapath
            output_filepath = output_dest_dir + os.sep + filename
            dfs_output_filepath = dfs_output_dir + ma.const.dfs_dir_sep + filename  
            
            print(filename, output_filepath, dfs_output_filepath)
            
            # copy file to hdfs if present
            if os.path.isfile(output_filepath):
                self.hdfs.delete_file(dfs_output_filepath, True)
                self.hdfs.copy_file_from_local(output_filepath, dfs_output_dir, job_id)
                self.__log.info("--> Copied final output to DFS: reduce task_id %d of job_id %d", reduce_id, job_id)
                print('COPIED COPIED OUTPUT', filename)
    
    
    def assignNewTasksToNT(self, tasks_to_run, actions):
        
        #if NT has requested for tasks to run as it has capacity to run more
        if tasks_to_run > 0:
            
            '''the following bit of code will get 
            1) a preference list of tasks from the TaskScheduler
            2) try and book these tasks for this particular NT
            3) make ReduceTip and MapTip of these tasks 
            objects to be passed to the NT to run as action objects!
            '''
            
            #get job_id and map_or_red tasks flag from the TasksScheduler
            #for that job
            job_id, map_or_red = self.task_scheduler.scheduleTasks()
             
            print("-------- Scheduler suggested:", job_id, map_or_red)
            
            if job_id == -1:
                #means no job in job list
                return
                        
            if map_or_red == MAP:
                list_of_tasks_assigned, maps_complete = self.dfsflags.choose_map_tasks(tasks_to_run, job_id)
                #if choose map tasks could not be done properly
                if list_of_tasks_assigned == None:
                    return
                if maps_complete == True:
                    self.task_scheduler.markMapsComplete(job_id)
                #task info tuple contains (map_id, struct_id, [inputs (filename,offset, size)])
                for task_info in list_of_tasks_assigned:
                
                    #GET MAP TASK SPECIFIC INFO FROM ITS XML
                    map_task = NewMapTaskAction(task_info[0], job_id, task_info[2], task_info[1])
                    #ADD TO ACTIONS LIST
                    actions.append(map_task)
            
                    self.__log.info("map task_id %s assigned by the HDFS of job_id %d", str(task_info[0]), job_id)
                    task_id = str(task_info[0])+'_'+str(job_id)+'_M'
                    
                 
            else: #if task is a reduce
                list_of_tasks_assigned, reduce_phase_completed, job_done = self.dfsflags.choose_reduce_tasks(tasks_to_run, job_id)
                
                print('--- REDUCES COMPLETED - ', reduce_phase_completed)
                
                # quitting MR, everything complete
                if job_done:
                    print('/////////////////// MR+ job assigned complete ///////////////////////')
                    print('/////////////////////////////////////////////////////////////////////')
                    print('/////////////////////      Exiting MR+       ////////////////////////')
                    # copy final output to DFS
                    self.finalOuputCopyToDfs(job_id, reduce_phase_completed)
                    print('current time:', time.time())
                    # kill the timer
                    self.__nt.timer.killTimerThread()
                    sys.exit(0)
                    os.system('killall -9 python')
                
                #if choose reduce tasks could not be done properly
                if list_of_tasks_assigned == None:
                    return
                
                #list has tuples (reduce_id, struct_id, level, [inputs ([MR], task_id)])
                for task_info in list_of_tasks_assigned:
                    #GET MAP TASK SPECIFIC INFO FROM ITS XML
                    
                    # inserting job id in tuple of input data
                    input_data = []
                    for i in task_info[3]:
                        input_data.append((job_id, i[0], i[1]))
                        
                    reduce_task = NewReduceTaskAction(job_id, task_info[0], input_data, task_info[2], task_info[1])
                    #ADD TO ACTIONS LIST
                    actions.append(reduce_task)
            
                    self.__log.info("reduce task_id %s assigned by the HDFS of job_id %d", str(task_info[0]), job_id)
                    task_id = str(task_info[0])+'_'+str(job_id)+'_R'
                    #self.running_tasks[task_id] = reduce_task


    def transferred_bytes_dfs(self):
        """This function returns the number of bytes transferred to and from
        the HDFS for Flags. This function cannot be used if the same HDFS
        object is being used as the nodetracker
        """
        return self.hdfs.bytes_copied_to_dfs() + self.hdfs.bytes_copied_to_local()
    
    